/**
 * 
 */
package com.ramon.heritrix;

import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.net.MalformedURLException;
import java.security.NoSuchAlgorithmException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;

import org.archive.io.arc.ARCReader;
import org.archive.io.arc.ARCReaderFactory;
import org.archive.io.arc.ARCRecord;

import com.ramon.ohcda.util.MD5;

/**
 * @author Ramon
 */
public class ArcParser {

    /**
     * @param args
     * @throws IOException
     * @throws MalformedURLException
     */

    private ByteArrayOutputStream bao;

    private String url;

    private ARCReader reader;

    private List<String> urlList;

    private HashMap<String, Object> hm;

    private Date now;

    private SimpleDateFormat formatter;

    private static boolean debug = false;

    private static MD5 md5;

    public ArcParser(String fileName) throws MalformedURLException,
            IOException, NoSuchAlgorithmException {

        urlList = new LinkedList<String>();
        hm = new HashMap<String, Object>();
        reader = ARCReaderFactory.get(fileName);
        formatter = new SimpleDateFormat("hh:mm:ss");
        md5 = MD5.getInstance();
        bao = new ByteArrayOutputStream();
    }

    public ArcParser(File f) throws MalformedURLException, IOException, NoSuchAlgorithmException {
        bao = new ByteArrayOutputStream();
        urlList = new LinkedList<String>();
        hm = new HashMap<String, Object>();
        reader = ARCReaderFactory.get(f);
        bao = new ByteArrayOutputStream();
        md5 = MD5.getInstance();
    }

    public void parse() throws IOException {
        for (final Iterator i = reader.iterator(); i.hasNext();) {
            ARCRecord ar = (ARCRecord) i.next();
            url = ar.getMetaData().getUrl();
            print(url + " length:" + ar.getMetaData().getLength() + " type:"
                    + ar.getMetaData().getMimetype());

            if (ar.getMetaData().getMimetype().equals("text/html")) {
                urlList.add(url);              
                ar.dump(bao);
                try {
                    hm.put(url, md5.hashData(bao.toByteArray()));
                } catch (final java.lang.OutOfMemoryError e) {
                    System.out.println(url + " is too large");
                }
                bao.reset();
            }
        }
    }

    public HashMap<String, Object> getHashMap() {
        return this.hm;
    }

    public List<String> getUrlList() {
        return this.urlList;
    }

    public void destroy() {
        this.bao = null;
        this.hm = null;
        this.reader = null;
        this.url = null;
        this.urlList = null;
    }

    public void setDebug(boolean debug) {
        this.debug = debug;
    }

    private void print(String msg) {
        if (debug) {
            now = Calendar.getInstance().getTime();
            System.out.println("[" + formatter.format(now) + "] " + msg);
        }
    }

    public static void main(String[] args) throws MalformedURLException,
            IOException, NoSuchAlgorithmException {
        ArcParser ap = new ArcParser(
                "data/IAH-20090601155137-00004-GillBates.arc.gz");
        ap.setDebug(true);
        ap.parse();

    }
}
